#include "mitrigger.h"

mitrigger::mitrigger(void) {
	triggerTable = NULL;
	htbTrigger = NULL;
	limSizeTrigger = 0;
	totalTrigger = 0;

	dictTable = NULL;
	htbDict = NULL;
	limSizeDict = 0;
	totalSoloToken = 0;

	dict = NULL;
	offset = 0;
	maxWindowSize = 0;
}

mitrigger::mitrigger( const string &inputFilename, dictionary &_dict, int _offset, int _maxWindowSize ) {
	ifstream inputFile(inputFilename.c_str());
	if (!inputFile) {
		cerr << "Cannot open input file " << inputFilename << ". Exiting..." << endl;
		exit(1);
	}
	
	dict = &_dict;
	offset = _offset;
	maxWindowSize = _maxWindowSize;

	char buffer[30];
	inputFile >> setw(30) >> buffer;
	inputFile.close();

	if (strncmp(buffer,"MITRIGGER",9) == 0)
		load(inputFilename);
	else
		generate(inputFilename);
}

mitrigger::~mitrigger(void)
{
	delete htbTrigger;
	delete[] triggerTable;

	delete htbDict;
	delete[] dictTable;
	
	//delete st;
}

double mitrigger::logAdd( const double lx, const double ly ) {
	if (lx == neg_inf) return ly;
	if (ly == neg_inf) return lx;
	double d = lx - ly;
	if (d >= 0) {
		if (d > 50) {
			return lx;
		} else {
			return (double)(lx + log(1.0 + exp(-d)));
		}
	} else {
		if (d < -50) {
			return ly;
		} else {
			return (double)(ly + log(1.0 + exp(d)));
		}
	}
}

double mitrigger::logSubtract( const double lx, const double ly ) {
	if (lx == ly) {
		return neg_inf;
	} else if (lx > ly) {
		return (double)(lx + log(1.0 - exp(ly - lx)));
	} else {
		return (double)(ly + log(1.0 - exp(lx - ly)));
	}
}

void mitrigger::tokenize( const string& str, const char& delimiter, vector<string>& tokens ) {
	// Skip delimiters at beginning. Find first "non-delimiter".
	string::size_type lastPos = str.find_first_not_of(delimiter, 0);
	// Find next instance of the delimiter.
	string::size_type pos = str.find_first_of(delimiter, lastPos);

	while (string::npos != pos || string::npos != lastPos) {
		// Add the token from lastPos to pos.
		tokens.push_back(str.substr(lastPos, pos - lastPos));

		// Find the start of the next token.
		lastPos = str.find_first_not_of(delimiter, pos);

		// Find the end of the token.
		pos = str.find_first_of(delimiter, lastPos);
	}
}

void mitrigger::growTriggerTable() {
	delete htbTrigger;

	cerr << "A+\b";

	unsigned int newlim = (unsigned int) (limSizeTrigger * GROWTH_STEP);
	trigger_entry *newTriggerTable = new trigger_entry[newlim];

	memcpy(newTriggerTable, triggerTable, sizeof(trigger_entry) * limSizeTrigger);

	delete[] triggerTable;
	triggerTable = newTriggerTable;

	htbTrigger = new htable<unsigned int*>((size_t) (newlim / DICTIONARY_LOAD_FACTOR));
	htbTrigger->set_keylen(4);
	for (unsigned int i = 0; i < limSizeTrigger; ++i) {
		htbTrigger->insert((unsigned int*) &triggerTable[i].wCode);
	}

	limSizeTrigger = newlim;
}

void mitrigger::growDictTable() {
	delete htbDict;

	cerr << "D+\b";
	unsigned int newlim = (unsigned int) (limSizeDict * GROWTH_STEP);
	dict_entry_mi *newDictTable = new dict_entry_mi[newlim];

	memcpy(newDictTable, dictTable, sizeof(dict_entry_mi) * limSizeDict);

	delete[] dictTable;
	dictTable = newDictTable;

	htbDict = new htable<unsigned int*>((size_t) (newlim / DICTIONARY_LOAD_FACTOR));
	htbDict->set_keylen(4);
	for (unsigned int i = 0; i < limSizeDict; ++i) {
		htbDict->insert((unsigned int*) &dictTable[i].wCode);
	}

	limSizeDict = newlim;
}

unsigned int mitrigger::size() {
	unsigned int result = 0;
	for (unsigned int i = 0; i < totalTrigger; i++)
	{
		result += triggerTable[i].triggeredTable->size();
	}
	return result;
}

void mitrigger::generate( const string &inputFilename) {
	ifstream inputFile(inputFilename.c_str());
	if (!inputFile) {
		cerr << "Cannot open input file " << inputFilename << ". Exiting..." << endl;
		exit(1);
	}

	// ---
	totalSoloToken = 0; //!< number of dict entry
	limSizeDict = 5000;
	dictTable = new dict_entry_mi[limSizeDict];
	htbDict = new htable<unsigned int*>((size_t) (limSizeDict / DICTIONARY_LOAD_FACTOR));
	htbDict->set_keylen(4);

	totalTrigger = 0; // !< number of trigger
	limSizeTrigger = 5000;
	triggerTable = new trigger_entry[limSizeTrigger];
	htbTrigger = new htable<unsigned int*>((size_t) (limSizeTrigger / DICTIONARY_LOAD_FACTOR));
	htbTrigger->set_keylen(4);

	totalTriggerLoad = 0;
	limSizeTriggerLoad = 0;
	triggerTableLoad = NULL;
	htbTriggerLoad = NULL;

	totalSoloCount = 0;
	totalPairCount = 0;

	dict_entry_mi *ptrDict;
	trigger_entry *ptrTrigger;
	unsigned int *pToken1;
	unsigned int *pToken2;

	clock_t clock1, clock2;
	// ---
	clock1 = clock();

	cerr << "Reading input file and computing counts." << endl;
	string line;
	unsigned int curr = 0;
	//unsigned int filterIndex = 0;
	// Go through all lines in the input file.
	while (!inputFile.eof()) {
		getline(inputFile, line);
		if (line != "") {
			++curr;
			if (curr % 20000 == 0) {
				cerr << ".";
				if (curr % 100000 == 0) {
					clock2 = clock();
					double partialTime = (((double)(clock2 - clock1)) / ((double)(CLOCKS_PER_SEC)));
					cerr << " processed " << curr << " lines (" << partialTime << " sec) " << endl;
				}
			}

			// Tokenize the line.
			vector<string> tokens;
			tokenize(line, ' ', tokens);
			// Add the total number of tokens in the line to our token count.
			totalSoloCount += tokens.size();
			// Go through all tokens and increment the unigram and pair counts.
			for (unsigned int i = 0; i < tokens.size(); ++i) {
				for (unsigned int j = i + offset + 1; j < min((unsigned int) (i + maxWindowSize + 1), (unsigned int) tokens.size()); ++j) {
					if (dict->findEntry(tokens[i]) && dict->findEntry(tokens[j])) {
						// ---
						// Trigger
						pToken1 = dict->encode(tokens[i]);
						ptrDict = (dict_entry_mi *)htbDict->find((unsigned int*) &pToken1);
						if (ptrDict != NULL) {
							dictTable[ptrDict->code].first += 1;
							pToken1 = dictTable[ptrDict->code].wCode;
						} else {
							dictTable[totalSoloToken].wCode = dict->encode(tokens[i]);
							pToken1 = dictTable[totalSoloToken].wCode;
							htbDict->insert((unsigned int *)&dictTable[totalSoloToken].wCode);
							dictTable[totalSoloToken].code = totalSoloToken;
							dictTable[totalSoloToken].first = 1;
							dictTable[totalSoloToken].second = 0;
							++totalSoloToken;
							// Check number of token
							if (totalSoloToken == limSizeDict) {
								growDictTable();
							}
						}

						// Triggered
						pToken2 = dict->encode(tokens[j]);
						ptrDict = (dict_entry_mi *)htbDict->find((unsigned int *) &pToken2);
						if (ptrDict != NULL) {
							dictTable[ptrDict->code].second += 1;
							pToken2 = dictTable[ptrDict->code].wCode;
						} else {
							dictTable[totalSoloToken].wCode = dict->encode(tokens[j]);
							pToken2 = dictTable[totalSoloToken].wCode;
							htbDict->insert((unsigned int *)&dictTable[totalSoloToken].wCode);
							dictTable[totalSoloToken].code = totalSoloToken;
							dictTable[totalSoloToken].first = 0;
							dictTable[totalSoloToken].second = 1;
							++totalSoloToken;
							// Check number of token
							if (totalSoloToken == limSizeDict) {
								growDictTable();
							}
						}

						// Pair
						ptrTrigger = (trigger_entry *) htbTrigger->find((unsigned int *) &pToken1);
						if (ptrTrigger != NULL) { // Exist trigger
							unsigned int indexTrigger = ptrTrigger->code;
							(*triggerTable[indexTrigger].triggeredTable)[pToken2] += 1;
						} else { // Don't exist trigger
							// Create new trigger
							triggerTable[totalTrigger].wCode = pToken1;
							htbTrigger->insert((unsigned int *) &triggerTable[totalTrigger].wCode);
							triggerTable[totalTrigger].code = totalTrigger;
							triggerTable[totalTrigger].triggeredTable =  new map<unsigned int*, unsigned long long>();
							// Create new triggered
							(*triggerTable[totalTrigger].triggeredTable)[pToken2] += 1;
							// Increase total trigger
							totalTrigger++;

							// Check number of trigger
							if (totalTrigger == limSizeTrigger) {
								growTriggerTable();
							}
						}

						// ---
						++totalPairCount;
					}
				}
			}
			tokens.clear();
		}
		line.clear();
	}

	//delete dict;
	clock2 = clock();
	double readInputTime = (((double)(clock2 - clock1)) / ((double)(CLOCKS_PER_SEC)));
	//cerr << "\n" << totalPairCount << "\nDone (" << readInputTime << " sec)" << endl << endl;
	cerr << "Done (" << readInputTime << " sec)" << endl << endl;
}

void mitrigger::load( const string &inputFilename) {
	ifstream inputFile(inputFilename.c_str(), ios_base::binary);
	if (!inputFile) {
		cerr << "Cannot open input file " << inputFilename << ". Exiting..." << endl;
		exit(1);
	}

	// ---
	totalSoloToken = 0; //!< number of dict entry
	limSizeDict = 0;
	dictTable = NULL;
	htbDict = NULL;

	totalTrigger = 0; // !< number of trigger
	limSizeTrigger = 0;
	triggerTable = NULL;
	htbTrigger = NULL;

	totalTriggerLoad = 0;
	limSizeTriggerLoad = 2000;
	triggerTableLoad = new trigger_entry_load[limSizeTriggerLoad];
	htbTriggerLoad = new htable<unsigned int*>((size_t) (limSizeTriggerLoad / DICTIONARY_LOAD_FACTOR));
	htbTriggerLoad->set_keylen(4);

	trigger_entry_load *ptrTriggerLoad;
	unsigned int *pToken1 = new unsigned int;
	unsigned int *pToken2 = new unsigned int;
	double mi;
	double pmi;
	//char *trigger, *triggered;

	clock_t clock1, clock2;
	// ---

	clock1 = clock();

	cerr << "Loading trigger pairs." << endl;

	// Binary file
	char header[20];
	inputFile.read(header, 10); header[10] = '\0';
	unsigned int line=1;

	inputFile.read(reinterpret_cast<char*>(&offset), sizeof(int));
	inputFile.read(reinterpret_cast<char*>(&maxWindowSize), sizeof(int));

	while (!inputFile.eof()) {
		inputFile.read(reinterpret_cast<char*>(&mi), sizeof(double));
		inputFile.read(reinterpret_cast<char*>(&pmi), sizeof(double));
		inputFile.read(reinterpret_cast<char*>(pToken1), sizeof(unsigned int));
		inputFile.read(reinterpret_cast<char*>(pToken2), sizeof(unsigned int));
		line++;

		// ---
		unsigned int *tmp1 = dict->encode((*dict)[*pToken1]);
		unsigned int *tmp2 = dict->encode((*dict)[*pToken2]);
		ptrTriggerLoad = (trigger_entry_load *) htbTriggerLoad->find((unsigned int*) &tmp1);
		if (ptrTriggerLoad != NULL) {
			unsigned int idx = ptrTriggerLoad->code;
			(*triggerTableLoad[idx].triggeredTable)[tmp2] = make_pair(mi,pmi);
		} else {
			triggerTableLoad[totalTriggerLoad].wCode = dict->encode((*dict)[*pToken1]);
			htbTriggerLoad->insert((unsigned int*)&triggerTableLoad[totalTriggerLoad].wCode);
			triggerTableLoad[totalTriggerLoad].code = totalTriggerLoad;
			triggerTableLoad[totalTriggerLoad].triggeredTable = new map<unsigned int*, pair<double, double> >() ;
			(*triggerTableLoad[totalTriggerLoad].triggeredTable)[tmp2] = make_pair(mi, pmi);
			++totalTriggerLoad;
			// Check number of trigger
			if (totalTriggerLoad == limSizeTriggerLoad) {
				growTriggerTableLoad();
			}
		}
		// ---
		if (line % 100000 == 0) {
			cerr << ".";
		}
	}
	clock2 = clock();
	double readInputTime = (((double)(clock2 - clock1)) / ((double)(CLOCKS_PER_SEC)));
	cerr << "\nDone (" << readInputTime << " sec)" << endl << endl;
}

void mitrigger::compMI( const string &outputFilename, double min_mi, int min_pair_count ) {
	ofstream outputFile(outputFilename.c_str(), ios_base::binary);
	if (!outputFile) {
		cerr << "Cannot open output file " << outputFilename << ". Exiting..." << endl;
		exit(1);
	}

	// ---
	clock_t clock2, clock3, clock4;
	int curr = 0;
	int numPrinted = 0;
	dict_entry_mi *ptrDict;
	map<unsigned int*, unsigned long long>::iterator itTriggered;

	// ---
	cerr << "Now computing mutual information and writing trigger pairs to output file." << endl;

	clock4 = clock();
	double numPairs = (double) (size());
	//cerr << "\n" << (int)numPairs << endl;
	outputFile << "MITRIGGER\n";
	outputFile.write(reinterpret_cast<char*>(&offset), sizeof(int));
	outputFile.write(reinterpret_cast<char*>(&maxWindowSize), sizeof(int));

	for (unsigned int i = 0; i < totalTrigger; ++i) {
		for (itTriggered = triggerTable[i].triggeredTable->begin(); itTriggered != triggerTable[i].triggeredTable->end(); ++itTriggered) {
			if (itTriggered->second < (unsigned long long)min_pair_count) {
				unsigned int *word1 = triggerTable[i].wCode;
				unsigned int *word2 = itTriggered->first;
				ptrDict = (dict_entry_mi *) htbDict->find((unsigned int*) &word1);
				assert(ptrDict != NULL);
				dictTable[ptrDict->code].first -= itTriggered->second;
				ptrDict = (dict_entry_mi *) htbDict->find((unsigned int*) &word2);
				assert(ptrDict != NULL);
				dictTable[ptrDict->code].second -= itTriggered->second;
				totalPairCount -= itTriggered->second;
			}
		}
	}
	//cerr << "\n" << totalPairCount << endl;
	//int db_Size = 0;
	double logTotalPairCount = log((double) totalPairCount);
	for (unsigned int i = 0; i < totalTrigger; ++i) {
		for (itTriggered = triggerTable[i].triggeredTable->begin(); itTriggered != triggerTable[i].triggeredTable->end(); ++itTriggered) {
			if (itTriggered->second >= (unsigned long long)min_pair_count) {
				//db_Size++;
				// Compute the mutual information between the two words in the pair.
				unsigned int *word1 = triggerTable[i].wCode;
				unsigned int *word2 = itTriggered->first;
				ptrDict = (dict_entry_mi *) htbDict->find((unsigned int*) &word1);
				assert(ptrDict != NULL);
				double logCount1 = log((double) dictTable[ptrDict->code].first);
				ptrDict = (dict_entry_mi *) htbDict->find((unsigned int*) &word2);
				assert(ptrDict != NULL);
				double logCount2 = log((double) dictTable[ptrDict->code].second);
				unsigned long long pairCount = itTriggered->second;
				double logPairCount = log((double)pairCount);
				double logCount1And2Inverse = logSubtract(logCount1, logPairCount);
				double logCount1InverseAnd2 = logSubtract(logCount2, logPairCount);
				double logCount1Inverse = logSubtract(logTotalPairCount, logCount1);
				double logCount2Inverse = logSubtract(logTotalPairCount, logCount2);
				double logCount1InverseAnd2Inverse = logSubtract(logSubtract(logAdd(logTotalPairCount, logPairCount), logCount1), logCount2);
				if (logPairCount != logPairCount) cerr << "Warning: logPairCount = " << logPairCount << endl;
				if (logCount1Inverse != logCount1Inverse) cerr << "Warning: logCount1Inverse = " << logCount1Inverse << endl;
				if (logCount2Inverse != logCount2Inverse) cerr << "Warning: logCount2Inverse = " << logCount2Inverse << endl;
				if (logCount1And2Inverse != logCount1And2Inverse) cerr << "Warning: logCount1And2Inverse = " << logCount1And2Inverse << endl;
				if (logCount1InverseAnd2 != logCount1InverseAnd2) cerr << "Warning: logCount1InverseAnd2 = " << logCount1InverseAnd2 << endl;
				if (logCount1InverseAnd2Inverse != logCount1InverseAnd2Inverse) 
					cerr << "Warning: logCount1InverseAnd2Inverse = " << logCount1InverseAnd2Inverse << endl;

				// P(A, B)log(P(A, B)/(P(A)P(B))
				double term1 = exp(logPairCount - logTotalPairCount);
				double term1LogPart = logPairCount + logTotalPairCount - (logCount1 + logCount2);
				term1 *= term1LogPart;

				// We have to explicitly check for neg_inf below to avoid nans since neg_inf * 0.0 = nan.
				// P(A, -B)log(P(A, -B)/(P(A)P(-B))
				double term2 = 0.0;
				if (logCount1And2Inverse != neg_inf) {
					term2 = exp(logCount1And2Inverse - logTotalPairCount);
					double term2LogPart = logCount1And2Inverse + logTotalPairCount - (logCount1 + logCount2Inverse);
					term2 *= term2LogPart;
				}

				// P(-A, B)log(P(-A, B)/(P(-A)P(B))
				double term3 = 0.0;
				if (logCount1InverseAnd2 != neg_inf) {
					term3 = exp(logCount1InverseAnd2 - logTotalPairCount);
					double term3LogPart = logCount1InverseAnd2 + logTotalPairCount - (logCount1Inverse + logCount2);
					term3 *= term3LogPart;
				}

				// P(-A, -B)log(P(-A, -B)/(P(-A)P(-B))
				double term4 = exp(logCount1InverseAnd2Inverse - logTotalPairCount);
				double term4LogPart = logCount1InverseAnd2Inverse + logTotalPairCount - (logCount1Inverse + logCount2Inverse);
				term4 *= term4LogPart;

				double mutualInformation = term1 + term2 + term3 + term4;
				//if (mutualInformation < 0.0) mutualInformation = 0.0;
				if (mutualInformation > min_mi) {
					if (term1LogPart > 0) {
						//outputFile << mutualInformation << "\t" << term1LogPart << "\t" << *word1 << "\t" << *word2 << "\t" << endl;
						outputFile.write(reinterpret_cast<char*>(&mutualInformation), sizeof(double));
						outputFile.write(reinterpret_cast<char*>(&term1LogPart), sizeof(double));
						outputFile.write(reinterpret_cast<char*>(&(*word1)), sizeof(unsigned int));
						outputFile.write(reinterpret_cast<char*>(&(*word2)), sizeof(unsigned int));
						outputFile.flush();
						++numPrinted;
					}
				}
				++curr;
				if (curr % 50000 == 0) {
					cerr << ".";
					if (curr % 1000000 == 0) {
						clock2 = clock();
						double partialTime = (((double)(clock2 - clock4)) / ((double)(CLOCKS_PER_SEC)));
						cerr << " processed " << curr << " pairs, printed " << numPrinted << "; " 
							<< 100.0 * (((double)curr) / numPairs) << "% done (" << partialTime << " sec) " << endl;
					}
				}
			}
		}
	}
	clock3 = clock();
	double printingTime = (((double)(clock3 - clock4)) / ((double)(CLOCKS_PER_SEC)));
	//cerr << db_Size << "\n100.00% done (" << printingTime << " sec)" << endl;
	cerr << "100.00% done (" << printingTime << " sec)" << endl;
	outputFile.close();
}

void mitrigger::stat( const string &outputFilename )
{
	ofstream outputFile(outputFilename.c_str());
	if (!outputFile) {
		cerr << "Cannot open output file " << outputFilename << ". Exiting..." << endl;
		exit(1);
	}

	//map<char*, int, classcomp>::iterator itTriggered;
	//// ---
	//for (int i = 0; i < totalTrigger; ++i) {
	//	for (itTriggered = triggerTable[i].triggeredTable->begin(); itTriggered != triggerTable[i].triggeredTable->end(); ++itTriggered) {
	//		if (itTriggered->second == 1) {
	//			outputFile << triggerTable[i].word << "\t" << itTriggered->first << endl;
	//		}
	//	}
	//}
	//// ---

	//// ---
	//map<int, int> mStat;
	//for (int i = 0; i < totalTrigger; ++i) {
	//	for (itTriggered = triggerTable[i].triggeredTable->begin(); itTriggered != triggerTable[i].triggeredTable->end(); ++itTriggered) {
	//		mStat[itTriggered->second] += 1;
	//	}
	//}

	//map<int, int>::iterator itStat;
	//outputFile << size() << "\t" << totalSoloToken << endl;
	//for (itStat = mStat.begin(); itStat != mStat.end(); ++itStat) {
	//	outputFile << itStat->first << "\t" << itStat->second << endl;
	//}
	//outputFile << "-----------\n";
	//mStat.clear();
	//for (int i = 0; i < totalSoloToken; ++i) {
	//	mStat[dictTable[i].first] += 1;
	//}
	//outputFile << "First:\n";
	//for (itStat = mStat.begin(); itStat != mStat.end(); ++itStat) {
	//	outputFile << itStat->first << "\t" << itStat->second << endl;
	//}
	//mStat.clear();
	//for (int i = 0; i < totalSoloToken; ++i) {
	//	mStat[dictTable[i].second] += 1;
	//}
	//outputFile << "Second:\n";
	//for (itStat = mStat.begin(); itStat != mStat.end(); ++itStat) {
	//	outputFile << itStat->first << "\t" << itStat->second << endl;
	//}
	//mStat.clear();
	//int iStat = 0;
	//int threshold = 5;
	//for (int i = 0; i < totalSoloToken; ++i) {
	//	if (dictTable[i].first < threshold && dictTable[i].second < threshold) {
	//		iStat++;
	//	}
	//}
	//outputFile << "Both < " << threshold << "\t" << iStat << endl;

	//outputFile << "DICT_ENTRY\t" << endl;
	//for (int i = 0; i < totalSoloToken; ++i) {
	//	outputFile << dictTable[i].word << endl;
	//}
	// ---
}

void mitrigger::growTriggerTableLoad() {
	delete htbTriggerLoad;

	cerr << "A+\b";

	unsigned int newlim = (unsigned int) (limSizeTriggerLoad * GROWTH_STEP);
	trigger_entry_load *newTriggerTableLoad = new trigger_entry_load[newlim];

	memcpy(newTriggerTableLoad, triggerTableLoad, sizeof(trigger_entry_load) * limSizeTriggerLoad);

	delete[] triggerTableLoad;
	triggerTableLoad = newTriggerTableLoad;

	htbTriggerLoad = new htable<unsigned int*>((size_t) (newlim / DICTIONARY_LOAD_FACTOR));
	htbTriggerLoad->set_keylen(4);
	for (unsigned int i = 0; i < limSizeTriggerLoad; ++i) {
		htbTriggerLoad->insert((unsigned int*)&triggerTableLoad[i].wCode);
	}

	limSizeTriggerLoad = newlim;
}

double mitrigger::scoreMI( const string &trigger, const string &triggered ) {
	if (triggerTableLoad != NULL) {
		unsigned int *tmp = dict->encode(trigger);
		trigger_entry_load *ptrTriggerLoad = (trigger_entry_load*)htbTriggerLoad->find((unsigned int*) &tmp);
		if (ptrTriggerLoad != NULL) {
			tmp = dict->encode(triggered);
			return (*triggerTableLoad[ptrTriggerLoad->code].triggeredTable)[dict->encode(triggered)].first;
		} else {
			return 0;
		}
	} else {
		cerr << "Error: \n";
		exit(1);
	}
}

double mitrigger::scoreMI( const char *trigger, const char *triggered ) {
	if (triggerTableLoad != NULL) {
		unsigned int *tmp = dict->encode(trigger);
		trigger_entry_load *ptrTriggerLoad = (trigger_entry_load*)htbTriggerLoad->find((unsigned int*) &tmp);
		if (ptrTriggerLoad != NULL) {
			return (*triggerTableLoad[ptrTriggerLoad->code].triggeredTable)[dict->encode(triggered)].first;
		} else {
			return 0;
		}
	} else {
		cerr << "Error: \n";
		exit(1);
	}
}

double mitrigger::scorePMI( const string &trigger, const string &triggered ) {

	if (triggerTableLoad != NULL) {
		unsigned int *tmp = dict->encode(trigger);
		trigger_entry_load *ptrTriggerLoad = (trigger_entry_load*)htbTriggerLoad->find((unsigned int*) &tmp);
		if (ptrTriggerLoad != NULL) {
			return (*triggerTableLoad[ptrTriggerLoad->code].triggeredTable)[dict->encode(triggered)].second;
		} else {
			return 0;
		}
	} else {
		cerr << "Error: \n";
		exit(1);
	}
}

double mitrigger::scorePMI( const char *trigger, const char *triggered ) {
	if (triggerTableLoad != NULL) {
		unsigned int *tmp = dict->encode(trigger);
		trigger_entry_load *ptrTriggerLoad = (trigger_entry_load *)htbTriggerLoad->find((unsigned int*) &tmp);
		if (ptrTriggerLoad != NULL) {
			return (*triggerTableLoad[ptrTriggerLoad->code].triggeredTable)[dict->encode(triggered)].second;
		} else {
			return 0;
		}
	} else {
		cerr << "Error: \n";
		exit(1);
	}
}

double mitrigger::GetPMIValue( const string &context, const string &word, int N )
{
	double score = 0.0;

	string strPartial = context + " " + word;
	vector<string> vecContext;
	vector<string> vecWords;
	tokenize(context, ' ', vecContext);
	tokenize(strPartial, ' ', vecWords);
	
	for (int i = (int)vecContext.size(); i < (int)vecWords.size(); i++) {
		for (int j = max((int) 0, (int) i - maxWindowSize); j < i - N + 1; j++) {
			score += scorePMI(vecWords[j], vecWords[i]);
		}
	}
	return score;
}

double mitrigger::GetLanguageCost( const string& words, int N )
{
	double score = 0.0;
	vector<string> vecWords;
	tokenize(words, ' ', vecWords);

	for (int i = N; i < (int)vecWords.size(); i++) {
		for (int j = max((int) 0, (int) i - maxWindowSize); j < i - N + 1; j++) {
			score += scorePMI(vecWords[j], vecWords[i]);
		}
	}

	return score;
}

double mitrigger::GetPhraseProb( const string &phrase, const int &N )
{
	double score = 0.0;
	vector<string> vecWords;
	tokenize(phrase, ' ', vecWords);

	for (int i = N; i < (int)vecWords.size(); i++)
	{
		for (int j = max((int) 0, (int) i - maxWindowSize); j < i - N + 1; j++) {
			score += scorePMI(vecWords[j], vecWords[i]);
		}
	}

	return score;
}

